{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "datasets: (60000, 28, 28) (60000,) 0 255\n"
     ]
    }
   ],
   "source": [
    "import  tensorflow as tf\n",
    "from    tensorflow.keras import datasets, layers, optimizers, Sequential, metrics\n",
    "(xs, ys),_ = datasets.mnist.load_data()\n",
    "print('datasets:', xs.shape, ys.shape, xs.min(), xs.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, ..., 5, 6, 8], dtype=uint8)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAOjUlEQVR4nO3df5BV9XnH8c8HXEDwR0QCUqRRKTGxaUXdSlrTjkaTUdMWjcaRtIa2ttBWa7Sa1LF/xH86ddqoSWxii9EGnUTHiXF0EqeRwVjq2CKrUsUQxViiwAa0jII2wrI+/WMvmY3u+e56f7PP+zWzc+89zz33PJzZD+fs/Z57v44IARj/JnS6AQDtQdiBJAg7kARhB5Ig7EASB7RzY5M8OaZoWjs3CaTypt7QntjtkWoNhd32mZK+LGmipK9HxHWl50/RNC306Y1sEkDBmlhVWav7NN72RElflXSWpOMkLbZ9XL2vB6C1Gvmb/WRJz0fECxGxR9JdkhY1py0AzdZI2OdIemnY4821Zb/A9lLbfbb7BrS7gc0BaEQjYR/pTYB3XHsbEcsjojciens0uYHNAWhEI2HfLGnusMdHStraWDsAWqWRsK+VNN/20bYnSbpQ0v3NaQtAs9U99BYRe21fKun7Ghp6uy0inmlaZwCaqqFx9oh4QNIDTeoFQAtxuSyQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgiYambLa9SdIuSYOS9kZEbzOaAtB8DYW95rSIeKUJrwOghTiNB5JoNOwh6UHbj9teOtITbC+13We7b0C7G9wcgHo1ehp/SkRstT1T0krbP4qI1cOfEBHLJS2XpEM8PRrcHoA6NXRkj4ittdvtku6VdHIzmgLQfHWH3fY02wfvuy/p45LWN6sxAM3VyGn8LEn32t73Ot+KiH9rSlcYN3zSr1bWBg+a1NBrT9pUHgTa+5OXGnr98abusEfEC5KOb2IvAFqIoTcgCcIOJEHYgSQIO5AEYQeSaMYHYbAf+9mi8nVQr84r/4qc+um1xfpVM/+lsjZn4tTiuqO56dVjivUHzzmxsja48YWGtr0/4sgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzj7OvXH+wmI9/vTlYv3JX/t2Q9v/3v/NrKw9NHhQQ6/90Wk/KtaXPPTDytriTy4rrhtrny7WD5h7ZLG++aaDi/VjZ2yvrL32kf8trlsvjuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7OPA9r/8rcraFZfdXVz3Dw6uHu+VpBOuv7RYP+TFwXL94ecra4OvNDae/KW/Or9Y/8oVX6us/fj88hj/+1/+5WL9+Hs3Fet/957y5/wvv7R6v04W4+wAGkDYgSQIO5AEYQeSIOxAEoQdSIKwA0k4Itq2sUM8PRb69LZtb7w44JijivVPfu+/KmsfnVo9zi1J537x88X6Ef/cV6zHwJ5ivZXcU57y+bmvf6iy9uwZtxTX/Z+9bxbrLw8eWKxfcmP5+oRZNz1arNdrTazSztjhkWqjHtlt32Z7u+31w5ZNt73S9sba7WHNbBhA843lNP4bks5827KrJa2KiPmSVtUeA+hio4Y9IlZL2vG2xYskrajdXyHpnCb3BaDJ6n2DblZE9EtS7bbyi8ZsL7XdZ7tvQLvr3ByARrX83fiIWB4RvRHR26PJrd4cgAr1hn2b7dmSVLstf3QKQMfVG/b7JS2p3V8i6b7mtAOgVUb9PLvtOyWdKmmG7c2SviDpOkl3275Y0ouSPtXKJrN78fxfKtYvPvSnlbUFf18eRx9tvLd9V2G8ey9d1VusbzzjnwrVEYeif+4vNi4u1ief/1qxPuvV1oyjN2LUsEdE1b+aq2OA/QiXywJJEHYgCcIOJEHYgSQIO5AEXyW9H5j0268U65v3vl5Zm7VmV7PbeVcmTJ1aWdtx3vHFdX/z8vLXMd9w+PXF+nMD1QOHF954VXHdOf+6vlgf3LmzWO9GHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2fcDv/7e/mL9tG99rrJ2zGP/2djGJ0wsln/2+ycV61Mv21JZe/TYrxbXXbu7/AHbRfdcUazPu7L6K7aPUPkjqOWJqPdPHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2fcDA2+Vx7o/dvqTlbVNMw4vrju449Vivf/yhcX6k1eWvq5Z2lsYsZ6/8s+L6x59R7Gseauqx9HxThzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtn3A4+u/UCx/qWzb6+sXXfGZ4rrHrL0pWL95qPK4+i/99zvFutvXH9kZW3+dx8rrovmGvXIbvs229ttrx+27FrbW2yvq/2c3do2ATRqLKfx35B05gjLb4yIBbWfB5rbFoBmGzXsEbFa0o429AKghRp5g+5S20/VTvMPq3qS7aW2+2z3DWh3A5sD0Ih6w36zpHmSFkjql1Q5w15ELI+I3ojo7dHkOjcHoFF1hT0itkXEYES8JekWSSc3ty0AzVZX2G3PHvbwXEnl+W0BdNyo4+y275R0qqQZtjdL+oKkU20vkBSSNkla1sIeMYpPTK2en/0T13+tuO5/vFn+Fbj2M39SrE94ZF2xPkVbi3W0z6hhj4jFIyy+tQW9AGghLpcFkiDsQBKEHUiCsANJEHYgCT7i2gYTpkwp1ndccEKxvvrcfxxlC1MrKwse+8PimnMueL5YnzBQHlrD/oMjO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kwTh7G2z6/InF+vpl5a9rvmPXMcX6RQf/tLK255lDi+vGwJ5iHeMHR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9ibY+JWF5fp55XH0D67+42L9V75Q/VXRkrTr3r7K2kHlGZmRCEd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfYxeuO86rH0Zac9VFz3A/9envb4/Z/bXldP+/zGgS9U1r69ZbCh18b4MeqR3fZc2z+wvcH2M7Y/W1s+3fZK2xtrt4e1vl0A9RrLafxeSVdGxAclfVjSJbaPk3S1pFURMV/SqtpjAF1q1LBHRH9EPFG7v0vSBklzJC2StKL2tBWSzmlVkwAa967eoLN9lKQTJK2RNCsi+qWh/xAkzaxYZ6ntPtt9A9rdWLcA6jbmsNs+SNI9ki6PiJ1jXS8ilkdEb0T09mhyPT0CaIIxhd12j4aC/s2I+E5t8Tbbs2v12ZIae0sZQEuNOvRm25JulbQhIm4YVrpf0hJJ19Vu72tJh11iy1nVQ1hXTX+2uO5d004q1vdu2VqsT5xxeLH+1O65lbXXl71aXHfKd4tljCNjGWc/RdJFkp62vW+y7ms0FPK7bV8s6UVJn2pNiwCaYdSwR8QjklxRPr257QBoFS6XBZIg7EAShB1IgrADSRB2IAk+4jpG73lyUnXxrPK6hx74ZkPbdk9PsT5v0rbK2uCDM0Z59efq6Aj7I47sQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AE4+xjNPv7/ZW1h/+6PA5+33F3FuvnrLywWL/4fQ8X68f2vFZZm/n4G8V1kQdHdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IwhHRto0d4umx0OPvC2l3fvrDxfpxl60v1g+cOFCsP/DYgmJ9/iVrinXksSZWaWfsGPHboDmyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASo46z254r6XZJR0h6S9LyiPiy7Wsl/Zmkl2tPvSYiHii91ngdZwe6RWmcfSxfXrFX0pUR8YTtgyU9bntlrXZjRHyxWY0CaJ2xzM/eL6m/dn+X7Q2S5rS6MQDN9a7+Zrd9lKQTJO27PvNS20/Zvs32YRXrLLXdZ7tvQLsbahZA/cYcdtsHSbpH0uURsVPSzZLmSVqgoSP/9SOtFxHLI6I3Inp7NLkJLQOox5jCbrtHQ0H/ZkR8R5IiYltEDEbEW5JukXRy69oE0KhRw27bkm6VtCEibhi2fPawp50rqfzRLgAdNZZ340+RdJGkp22vqy27RtJi2wskhaRNkpa1pEMATTGWd+MfkTTSuF1xTB1Ad+EKOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJtnbLZ9suSfjJs0QxJr7StgXenW3vr1r4keqtXM3t7X0S8d6RCW8P+jo3bfRHR27EGCrq1t27tS6K3erWrN07jgSQIO5BEp8O+vMPbL+nW3rq1L4ne6tWW3jr6NzuA9un0kR1AmxB2IImOhN32mbaftf287as70UMV25tsP217ne2+Dvdym+3tttcPWzbd9krbG2u3I86x16HerrW9pbbv1tk+u0O9zbX9A9sbbD9j+7O15R3dd4W+2rLf2v43u+2Jkp6T9DFJmyWtlbQ4In7Y1kYq2N4kqTciOn4Bhu3fkfS6pNsj4kO1Zf8gaUdEXFf7j/KwiPibLuntWkmvd3oa79psRbOHTzMu6RxJf6QO7rtCXxeoDfutE0f2kyU9HxEvRMQeSXdJWtSBPrpeRKyWtONtixdJWlG7v0JDvyxtV9FbV4iI/oh4onZ/l6R904x3dN8V+mqLToR9jqSXhj3erO6a7z0kPWj7cdtLO93MCGZFRL809MsjaWaH+3m7Uafxbqe3TTPeNfuununPG9WJsI80lVQ3jf+dEhEnSjpL0iW101WMzZim8W6XEaYZ7wr1Tn/eqE6EfbOkucMeHylpawf6GFFEbK3dbpd0r7pvKupt+2bQrd1u73A/P9dN03iPNM24umDfdXL6806Efa2k+baPtj1J0oWS7u9AH+9ge1rtjRPZnibp4+q+qajvl7Skdn+JpPs62Msv6JZpvKumGVeH913Hpz+PiLb/SDpbQ+/I/1jS33aih4q+jpH037WfZzrdm6Q7NXRaN6ChM6KLJR0uaZWkjbXb6V3U2x2Snpb0lIaCNbtDvX1EQ38aPiVpXe3n7E7vu0JfbdlvXC4LJMEVdEAShB1IgrADSRB2IAnCDiRB2IEkCDuQxP8DZudPCz43uygAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.imshow(xs[59999])\n",
    "ys\n",
    "#print (xs[0,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<RepeatDataset shapes: ((None, 28, 28), (None,)), types: (tf.float32, tf.uint8)>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = tf.convert_to_tensor(xs, dtype=tf.float32) / 255.\n",
    "db = tf.data.Dataset.from_tensor_slices((xs,ys))\n",
    "db = db.batch(32).repeat(10)\n",
    "db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28) (60000, 10)\n",
      "train_dataset:  <BatchDataset shapes: ((None, 28, 28), (None, 10)), types: (tf.float32, tf.float32)>\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "0 0 loss: 1.4168817\n",
      "y: tf.Tensor(\n",
      "[[0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " ...\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 1. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 1. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 1. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 1. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " ...\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " ...\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 1. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " ...\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 1.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " ...\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 1. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 1. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 1. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 1. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 1. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 1. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 1. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 1. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n",
      "y: tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(200, 10), dtype=float32)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-c7c26977d7c9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-39-c7c26977d7c9>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m         \u001b[0mtrain_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-39-c7c26977d7c9>\u001b[0m in \u001b[0;36mtrain_epoch\u001b[0;34m(epoch)\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;31m# Step3. optimize and update w1, w2, w3, b1, b2, b3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m         \u001b[0mgrads\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainable_variables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;31m# w' = w - lr * grad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainable_variables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/tf20/lib/python3.7/site-packages/tensorflow_core/python/eager/backprop.py\u001b[0m in \u001b[0;36mgradient\u001b[0;34m(self, target, sources, output_gradients, unconnected_gradients)\u001b[0m\n\u001b[1;32m   1012\u001b[0m         \u001b[0moutput_gradients\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_gradients\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m         \u001b[0msources_raw\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflat_sources_raw\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1014\u001b[0;31m         unconnected_gradients=unconnected_gradients)\n\u001b[0m\u001b[1;32m   1015\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_persistent\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/tf20/lib/python3.7/site-packages/tensorflow_core/python/eager/imperative_grad.py\u001b[0m in \u001b[0;36mimperative_grad\u001b[0;34m(tape, target, sources, output_gradients, sources_raw, unconnected_gradients)\u001b[0m\n\u001b[1;32m     74\u001b[0m       \u001b[0moutput_gradients\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m       \u001b[0msources_raw\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m       compat.as_str(unconnected_gradients.value))\n\u001b[0m",
      "\u001b[0;32m~/opt/anaconda3/envs/tf20/lib/python3.7/site-packages/tensorflow_core/python/eager/backprop.py\u001b[0m in \u001b[0;36m_gradient_function\u001b[0;34m(op_name, attr_tuple, num_inputs, inputs, outputs, out_grads, skip_input_indices)\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mnum_inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 138\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mgrad_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmock_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mout_grads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    139\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/tf20/lib/python3.7/site-packages/tensorflow_core/python/ops/math_grad.py\u001b[0m in \u001b[0;36m_MatMulGrad\u001b[0;34m(op, grad)\u001b[0m\n\u001b[1;32m   1573\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0m_MatMulGradAgainstFirstOnly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1574\u001b[0m       \u001b[0;32melif\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mskip_input_indices\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1575\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_MatMulGradAgainstSecondOnly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1576\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1577\u001b[0m     \u001b[0;31m# No gradient skipping, so do the full gradient computation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/tf20/lib/python3.7/site-packages/tensorflow_core/python/ops/math_grad.py\u001b[0m in \u001b[0;36m_MatMulGradAgainstSecondOnly\u001b[0;34m(op, grad)\u001b[0m\n\u001b[1;32m   1554\u001b[0m   \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1555\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mt_a\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mt_b\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1556\u001b[0;31m     \u001b[0mgrad_b\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgen_math_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmat_mul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtranspose_a\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1557\u001b[0m   \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mt_a\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mt_b\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1558\u001b[0m     \u001b[0mgrad_b\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgen_math_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmat_mul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtranspose_a\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/tf20/lib/python3.7/site-packages/tensorflow_core/python/ops/gen_math_ops.py\u001b[0m in \u001b[0;36mmat_mul\u001b[0;34m(a, b, transpose_a, transpose_b, name)\u001b[0m\n\u001b[1;32m   6110\u001b[0m         \u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_context_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_thread_local_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"MatMul\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6111\u001b[0m         \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_post_execution_callbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"transpose_a\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6112\u001b[0;31m         transpose_a, \"transpose_b\", transpose_b)\n\u001b[0m\u001b[1;32m   6113\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6114\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_FallbackException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import  os\n",
    "import  tensorflow as tf\n",
    "from    tensorflow import keras\n",
    "from    tensorflow.keras import layers, optimizers, datasets\n",
    "\n",
    "\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL']='2'\n",
    "\n",
    "(x, y), (x_val, y_val) = datasets.mnist.load_data() \n",
    "x = tf.convert_to_tensor(x, dtype=tf.float32) / 255.\n",
    "y = tf.convert_to_tensor(y, dtype=tf.int32)\n",
    "y = tf.one_hot(y, depth=10)\n",
    "print(x.shape, y.shape)\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((x, y))\n",
    "train_dataset = train_dataset.batch(200)\n",
    "\n",
    "print('train_dataset: ' ,train_dataset)\n",
    "\n",
    "model = keras.Sequential([ \n",
    "    layers.Dense(512, activation='relu'),\n",
    "    layers.Dense(256, activation='relu'),\n",
    "    layers.Dense(10)])\n",
    "\n",
    "optimizer = optimizers.SGD(learning_rate=0.001)\n",
    "\n",
    "\n",
    "def train_epoch(epoch):\n",
    "\n",
    "    # Step4.loop\n",
    "    for step, (x, y) in enumerate(train_dataset):\n",
    "    \n",
    "       # print('y:',y)\n",
    "\n",
    "        with tf.GradientTape() as tape:\n",
    "            # [b, 28, 28] => [b, 784]\n",
    "            x = tf.reshape(x, (-1, 28*28))\n",
    "            # Step1. compute output\n",
    "            # [b, 784] => [b, 10]\n",
    "            out = model(x)\n",
    "            # Step2. compute loss\n",
    "            loss = tf.reduce_sum(tf.square(out - y)) / x.shape[0]\n",
    "\n",
    "        # Step3. optimize and update w1, w2, w3, b1, b2, b3\n",
    "        grads = tape.gradient(loss, model.trainable_variables)\n",
    "        # w' = w - lr * grad\n",
    "        optimizer.apply_gradients(zip(grads, model.trainable_variables))\n",
    "\n",
    "        if step % 100 == 0:\n",
    "            print(epoch, step, 'loss:', loss.numpy())\n",
    "def train():\n",
    "\n",
    "    for epoch in range(30):\n",
    "\n",
    "        train_epoch(epoch)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
